#!/usr/bin/env ruby
############################################################################
#    Copyright (C) 2009 by Davide Monfrecola                               #
#    davide.monfrecola@gmail.com                                           #
#                                                                          #
#    This program is free software; you can redistribute it and#or modify  #
#    it under the terms of the GNU General Public License as published by  #
#    the Free Software Foundation; either version 2 of the License, or     #
#    (at your option) any later version.                                   #
#                                                                          #
#    This program is distributed in the hope that it will be useful,       #
#    but WITHOUT ANY WARRANTY; without even the implied warranty of        #
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the         #
#    GNU General Public License for more details.                          #
#                                                                          #
#    You should have received a copy of the GNU General Public License     #
#    along with this program; if not, write to the                         #
#    Free Software Foundation, Inc.,                                       #
#    59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.             #
############################################################################

=begin
This class of object contains all the information about a master document
There's one instance of this class for the master document
=end

class MasterDocument < WebDocument
  
	def initialize(url, copy, glade)
		
    if (super(url) == nil)
			return nil
		end
    
		@overlaps = []														# init overlaps array
		if (glade != nil)													# if plagtion is running without a GUI glade = nil
			progress = glade["progressbar1"]							# link to the progress bar GUI object
			progressLabel = glade["infoLabel3"]						# link to the information label GUI object 
		end #if
		@resultList = [] 													# -> for test only. contains Document object
		resultUrlList = []
		i = 0
		#puts "=== blockHash (Document)===" 							# calculate the block hash and populate the indexTable
			while i < @content.length-1 
				wlist = get_words(i, @bsize)
				hashValue = blockhash(wlist)  			
				@indexTable[hashValue] << i 							# i is the index of the first word in @content
				i = i+@bsize
			end # while
		# search with Google and Yahoo
		if (glade != nil)													# if plagtion is running without a GUI glade == nil
			Gtkgui.updateProgressBar(progress, "10 % Complete", 10.0 / 100.0)	
			if $local == true
				progressLabel.text = "Searching on local document"
			else
				progressLabel.text = "Searching on the web"
			end # if
			fraction = 10.0 												# init value of fraction
			step = 10.0 / $num_of_searches.to_f						# step value for searching sector
			Gtkgui.process_gui_events
		end #if
		##############
		# Local case #
		##############
		if $local == true
			#resultUrlList << glade["copyDocumentEntry"].text
			resultUrlList << copy
			#puts resultUrlList.size
			document = WebDocument.new(copy)
			if document != nil
				@resultList << document
				puts "* Document Object from #{document.doc_name} created *"
			else
				puts "Empty document not added" 
			end #if
		#################
		# Internet case #
		#################
		else
			($num_of_searches).times do
				begin	
					
          if $yahoo == true
						puts "=== searching on #{$num_of_searches} random block (Yahoo) ==="
						# Yahoo search #
						yahoo = YahooSearchEngine.new(self, @content, @bsize)
						resultUrlList << yahoo.search($num_of_results_yahoo)
						if (glade != nil)													# if plagtion is running without a GUI glade = nil
							fraction = fraction + step
							progress.fraction = fraction / 100.0
							progress.text = "#{fraction} % complete"
						end #if
					end # yahoo
          
					if $google == true
						puts "=== searching on #{$num_of_searches} random block (Google) ==="
						# Google search #
						google = GoogleSearchEngine.new(self, @content, @bsize)
						resultUrlList << google.search($num_of_pages) 			# return an UrlManager obj			
						if (glade != nil)													# if plagtion is running without a GUI glade = nil 
							fraction = fraction + step
							Gtkgui.updateProgressBar(progress, "#{fraction} % complete", fraction / 100.0)	
						end #if
					end # google
          
				rescue StandardError => msg
					$error_logger.error("MasterDocument") {msg}
				end #rescue
			end # NUM_OF_SEARCHS.times
			$logger.debug("MasterDocument") {"Url list size: #{resultUrlList.size}"}
				if (glade != nil)
					progressLabel.text = "Create Document objects from URLs"
					Gtkgui.updateProgressBar(progress, "20 % complete", 20.0 / 100.0)
				end #if
			for results in resultUrlList
				fetch_url(self, results, glade)
			end #for
		end # if (local)
end #init

=begin	
Search if there are a common region between self and doc (Document object)
to find common region it uses the Bentley McIlroy's algorithm
If one is found method "block_control" is called
Return an Overlap object
- doc: WebDocument object
=end
	def search_overlaps(doc)
		flag = 0														# if zero call blockhash, else call Bentley McIlroy algorithm
		i = 0
		# create new Overlap object that contains the information
		# on the common region found in the copy document
		overlap = Overlap.new(self, doc) 								# self -> MasterDocument | doc -> WebDocument
		
		while i < (doc.content).length
			wlist = doc.get_words(i, @bsize)							# get a words block from MasterDocument and put it in wlist
			
			if (wlist.size < @bsize)									# last block may be smaller then 5 words
				if (overlap.num_overlaps > 0)
					return overlap
				else
					return nil
				end #if
			end #if
			
			if (i == 0) || (flag == 0)
				index = blockhash(wlist) 
			else
				index = blockhash_Bentley(wlist) 		# This method use Bentley McIlroy algorithm
			end #if
			if (search_hashValue(index) == true)
				# search for possible hash collision (using block_control) 
				for y in 0...@indexTable[index].size
					begin 
						# MasterDocument @indexTable
						if (block_control(doc, wlist, @indexTable[index][y], i)) 					# true if find a copied block 
							size = @extended_index["end_copy"] - @extended_index["start_copy"]		# size of common block found
							#puts "!!== block of #{size} words found ==!!"
							overlap.add(size, @extended_index) 			# add the overlap region that has just found
							i += size
							flag = 0
						else
							# continue the search  
							i = i+1
							flag = 1
						end #if
					rescue Readers::ReaderError => msg
		      	# send message to log file
		      	$error_logger.error(msg)
		      	return nil
		   	 end #rescue
				end #for
			else
				i += 1
				flag = 1
			end #if
			
		end #while
		if (overlap.num_overlaps > 0)
			return overlap
		else
			return nil
		end #if
	end #search_overlaps
	
=begin
Searching for overlaps on documents found on the internet
=end
	def search_common_region(glade)
		threads = []																				# thread list
		if (glade != nil) 																# if plagtion is running without a GUI glade = nil
				progress = glade["progressbar1"]								# link to the progress bar GUI object
				progressLabel = glade["infoLabel3"]							# link to the information label GUI object
				progressLabel.text = "Searching for overlaps"		# change information label text 
				fraction = (progress.fraction) * 100.0					# init value of fraction
				step = (30.0 / (@resultList.size).to_f)					# step value for creating object sector (+50%)
				Gtkgui.process_gui_events
		end	# if
		for doc in @resultList															# Searchs if there are common regions fo r every Document object created
				if (glade != nil)
					fraction = fraction + step
					Gtkgui.updateProgressBar(progress, "#{fraction.to_i} % complete", fraction / 100.0)
				end #if
				puts "==== Searching overlaps on #{doc.doc_name} ===="
				overlap = self.search_overlaps(doc)						
				if overlap != nil															# if ovelap == nil no overlaps were found
					@overlaps << overlap
					puts "overlap added"
				end #if
		end #for
		puts "overlaps size: #{@overlaps.size}"
	end
	
=begin
Displays all the overlaps found
Return the number of overlaps found
=end
	def display_overlaps()
		i = 1
		j = 1
		print "\n==== Diplay overlaps (display_overlaps) ====\n"
		begin
			if (@overlaps != nil)
				if @overlaps.size > 0
					for item in @overlaps
						if item != nil
							print "\n*********************\n"
							print "MasterDocument name -> #{item.master_doc.doc_name}\n"
							print "CopyDoc ID -> #{item.copy_doc.doc_name}\n"
							puts "Total overlaps: #{item.num_overlaps}"
							puts "Total common words: #{item.tot_words}"

							item.overlaps.each do |index| 
								puts "Overlap size -> #{index[0]} - indexes: Master(#{index[1][0]}, #{index[1][1]}) Copy(#{index[2][0]}, #{index[2][1]})"
							end # do
							i += 1
							j = 0
						end # if
					end #for
					print "\n*********************\n"
				else
					print"=== No overlaps found ===\n\n"
				end #if
			end #if
		rescue StandardError, Errno => msg
			$error_logger.error("MasterDocument#display_overlaps") {msg}
		end #rescue
		print "\n"
		print "Collision detected: #{@@collision}\n"
		return @overlaps.size
	end #display

	# Return the @overlaps array
	def get_overlaps()
		if (@overlaps.size > 0)
			return @overlaps 
		else
			return nil
		end
	end
	
	private # private method
	
=begin
Control if the found block hash is the same block 
Return the size of the found string
- wlist: word list
- index_first: index of the first word block in @content (MasterDocument)
- index_second: index of the first word block in doc.content (WebDocument)
=end
	def block_control(doc, wlist, index_first, index_second)
			wlist_master = get_words(index_first, @bsize) 								# wlist contains the block
			if (wlist_master == wlist)
				extend_block(doc, wlist_master,index_first, index_second)
				return true
			else
				@@collision += 1
			end #if
			return false 																									# if the found block it's not the same return false (collision)
	end		
	
=begin
Right and left block extension
Used to find block that are larger than @bsize words
- i: index in @content (Masterdocument)
- j: index in doc.content (WebDocument)
=end
	def extend_block(doc, wlist_master, i, j)
		@extended_index = {} 		# hash with key "start_master", "start_copy", "end_master" and "end_copy"
		extension_i = i-1
		extension_j = j-1
		# left extension
		while (@content[extension_i][0] == doc.content[extension_j][0])
			extension_i -= 1
			extension_j -= 1
			break if (extension_i < 0) || (extension_j < 0)
		end  #while
		(extension_i < 0 ? @extended_index["start_master"] = 0 : @extended_index["start_master"] = extension_i+1)
		(extension_j < 0 ? @extended_index["start_copy"] = 0 : @extended_index["start_copy"] = extension_j+1)
		# right extension
		extension_i = i+@bsize
		extension_j = j+@bsize
		while (@content[extension_i][0] == doc.content[extension_j][0])
			extension_i += 1
			extension_j += 1
			break if (extension_i > @content.size-1) || (extension_j > doc.content.size-1)
		end # while
		@extended_index["end_master"] = extension_i
		@extended_index["end_copy"] = extension_j
		
	end # extend_block
	
=begin
Return true if @indexTable contains value
- value: hash of a block of the MasterDocument
=end
	def search_hashValue(value)
		return @indexTable[value][0] != nil
	end #search hash value
	
=begin
Fetchs URL in the list urlList
- urlList: contains all the urls find by Google
- master: MasterDocument pointer
- settings: settings hash
=end
	def fetch_url(master, urlList, glade)
		threads = []
		if (glade != nil) 																# if plagtion is running without a GUI glade = nil
			progress = glade["progressbar1"]								# link to the progress bar GUI object
			progressLabel = glade["infoLabel3"]							# link to the information label GUI object 
			fraction = (progress.fraction) * 100.0					# init value of fraction
			step = ((50.0 / ($num_of_searches*2).to_f) / (urlList.size).to_f)			# step value for creating object sector (+50%)
		end #if
		puts "==== Create Document objects from url ===="
		while ((url = urlList.get_next) != nil)
			threads << Thread.new do
				begin	
					document = WebDocument.new(url)
					if document != nil
						@mutex.lock                               # controls access to a shared source
						@resultList << document
						@mutex.unlock
					puts "* Document Object from #{document.doc_name} created *"
					else
						puts "Empty document not added" 
					end #if
					if (glade != nil)                           # if plagtion is running without a GUI glade = nil
						fraction = fraction + step
						Gtkgui.updateProgressBar(progress, "#{fraction.to_i} % complete", fraction / 100.0)
					end #if
				rescue StandardError, Timeout::Error => msg
					# write the error on a log file and continue the execution of program
					$error_logger.error("MasterDocument#fetch_url(Document.new)") {"Document creation failed -> #{msg} #{url}"}
					next                                        # jump to next iteration
				end #exception	
			end # do thread
		end #while
		threads.each {|t| t.join}													# wait the execution of every thread created
		puts "==== Document objects from url created ===="
		$logger.debug("MasterDocument#fetch_url") {"ResultList complete. Array size: #{@resultList.size}"}
	end # end fetch_url
	
end #class
